import csv
import time
import jsonpath
import requests


class SpiderHouse:
    def __init__(self):
        self.url = 'https://shuyang.anjuke.com/v3/ajax/map/sale/1718/prop_list/?room_num=-1&price_id=-1&area_id=-1&floor=-1&orientation=-1&is_two_years=0&is_school=0&is_metro=0&order_id=0&zoom=12&lat=34.032772_34.202051&lng=118.368882_119.251952&kw=&et=134b8b&ib=1&bst=pem989'

        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36'
        }

    def request_to_send(self):
        for num in range(1, 4):
            params = {
                'p': str(num)
            }
            r = requests.get(url=self.url, headers=self.headers, params=params, timeout=5).json()
            house_list = jsonpath.jsonpath(r, '$.val.props[:]')

            self.save_csv(house_list, num)

    def save_csv(self, house_list, num):
        # 组织表头
        dict_keys = ['region_name', 'block_name', 'rhval', 'area', 'price', 'long_title', 'comm_name',
                     'house_orient_name',
                     'floor_tag']
        # 除去数据中不需要的键值对，保留要保存的键值对
        csv_house_list = []
        for i in house_list:
            new_dict = dict()
            for x, y in i.items():
                if x in dict_keys:
                    new_dict[x] = y
            csv_house_list.append(new_dict)

        # 保存csv
        with open('house_info_page_' + str(num) + '.csv', 'w', encoding='utf-8') as f1:
            f_csv = csv.DictWriter(f1, dict_keys)
            f_csv.writeheader()
            f_csv.writerows(csv_house_list)
        print('第%s页抓取完成!共抓取%s条数据' % (str(num), str(len(csv_house_list))))
        time.sleep(3)


spider_house = SpiderHouse()
spider_house.request_to_send()
